package com.yaoandw.crawl;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.Date;
import java.util.StringTokenizer;

import com.yaoandw.crawl.model.CrawlUrl;
import com.yaoandw.crawl.storage.Storage;
import com.yaoandw.lucene.Page;
import com.yaoandw.lucene.PageIndexer;
import com.yaoandw.util.URLUtil;

public class MyCrawl implements Runnable{
	 public static final String[] patterns = {
	      "href=",
	      "src=",
	      "background="
	    };
//	 public static String indexDir = "D:/temp/study/java/lucene/firstluceneapp/pageindexer";
//	 public static String baseSite = "http://192.168.1.186/test";
	 public static String indexDir = "D:/temp/study/java/lucene/firstluceneapp/actizforumindexer";
	 public static String baseSite = "http://www.actiz.com:8080/bbs/";
	 private Storage storage ;
	 private PageIndexer pageIndexer ;
	 private int crawlLevel ;
	/**
	 * @param args
	 */
	public static void main(String[] args) {
		try {
			MyCrawl myCrawl = new MyCrawl(new URL(baseSite),true,2);
			Thread crawl = new Thread(myCrawl);
			crawl.start();
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	public void run() {
		try {
			crawl();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	private URL startUrl;
	public URL getStartURL() {
		return startUrl;
	}
	public MyCrawl(URL startUrl,boolean baseUrlOnly) throws IOException{
		this(startUrl,baseUrlOnly,1);
	}
	/**
	 * @param startUrl 开始地址
	 * @param baseUrlOnly 是否只趴开始网站
	 * @param level 深度
	 * @throws IOException
	 */
	public MyCrawl(URL startUrl,boolean baseUrlOnly,int level) throws IOException{
		this.startUrl = startUrl;
		this.crawlLevel = level;
		storage = Storage.getInstance(startUrl,baseUrlOnly,level);
		pageIndexer = new PageIndexer(indexDir);
	}
	
	public void crawl() throws IOException{
		long start = System.currentTimeMillis();
		storage.put(startUrl,0);
		try{
			while(storage.hasNext()){
				CrawlUrl url = storage.getNextUrl();
				if(url != null)
					crawl(url);
			}
		}finally{
			pageIndexer.close();
		}
		System.out.println("crawl size: " + storage.getCrawledUrl().size() + ", costs: " + (System.currentTimeMillis()-start)/1000 + "s");
	}
	public void crawl(CrawlUrl url){
		try{
			URL u = url.getUrl();
			URLConnection connection = u.openConnection();
			 int httpStatus = 0;
			if (connection instanceof HttpURLConnection) {
                httpStatus = ((HttpURLConnection) connection).getResponseCode();
                switch (httpStatus) {
	                case HttpURLConnection.HTTP_MOVED_PERM:
	                case HttpURLConnection.HTTP_MOVED_TEMP:
//	                    String redirectURL = connection.getHeaderField("location");
//	                    u = URLUtil.normalize(new URL(redirectURL));
//	                    break;
	                case HttpURLConnection.HTTP_NOT_FOUND:
	                	storage.deletePage(u.toString());
	                	return;
	                default:
	                    break;
                }
			}
			InputStreamReader isr = new InputStreamReader(u.openStream(),"GBK");
			BufferedReader br = new BufferedReader(isr);
			String s;
			StringBuffer sb = new StringBuffer();
			while ((s = br.readLine()) != null) {
	//			System.out.println(s);
				if(url.getLevel()<=crawlLevel);
					findURLs(this, s,url.getLevel());
				sb.append(s);
			}
			Page page = new Page("",sb.toString(),u.toString(),new Date());
			//create index
			if(storage.pageHasModified(page))
				pageIndexer.updateIndex(page );
		}catch(IOException e){
			e.printStackTrace();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	public static void findURLs(MyCrawl callback, String line,int level) {
        for (int i = 0; i < patterns.length; i++) {
            String pattern = patterns[i];
            findURLs(callback, line, pattern,level);
        }
    }
	protected static void findURLs(MyCrawl callback, String line, String pattern,int level) {
        String lineLowerCase = line.toLowerCase();
        int pos = lineLowerCase.indexOf(pattern);
        while (pos != -1) {
            String uri = "";
            try {
                uri = extractURL(line, pos + pattern.length());
                URL baseURL = callback.getStartURL();
                if ( ! URLUtil.isFileSpecified(baseURL)) {
                // Force a slash in case of a folder (to avoid buggy relative refs)
                   baseURL = new URL(baseURL.toString() + "/");
                }
                URL foundURL = URLUtil.normalize(new URL(baseURL, uri));
                callback.urlFound(foundURL,level);
            } catch (MalformedURLException e) {
                callback.malformedUrlFound(uri);
            }
            pos = lineLowerCase.indexOf(pattern, pos + pattern.length());
        }
    }

	private void malformedUrlFound(String uri) {
		// TODO Auto-generated method stub
//		System.out.println("malformedUrlFound: "+uri);
	}

	private void urlFound(URL foundURL,int level) {
		if(storage.put(foundURL,level + 1))
			System.out.println("urlFound: "+foundURL + ",level: " + (level + 1));
	}

	protected static String extractURL(String string, int pos) {
        char c = string.charAt(pos);
        String ret = "";
        if (c == '\'' || c == '"') {
            string = string.substring(pos + 1);
        } else {
            string = string.substring(pos);
        }
        if (string.length() > 0) {
            c = string.charAt(0);
            if (c == '\'' || c == '\"' || c == '>') {
                ret = "";
            } else {
                StringTokenizer st = new StringTokenizer(string, " \"\'>");
                ret = st.nextToken();
            }
        }
        int p = ret.indexOf('#');
        if (p > -1) {
            return ret.substring(0, p);
        } else {
            return ret;
        }
    }
}
